#!/usr/bin/env python
# coding=utf-8
"""
@Author  : youjia - 卞志伟
@file    : 测试爬虫.py
@contact : bianzhiwei@iyoujia.com
@time    : 2019-12-21 09:51 
@Desc    : 
@Software: PyCharm
"""
import os
import sys

# 当前文件的路径
pwd, filename = os.path.split(os.path.abspath(__file__))
# 当前文件的父路径
father_path = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".")
# 当前文件的前两级目录
grader_father = os.path.abspath(os.path.dirname(pwd) + os.path.sep + "..")
sys.path.append(pwd)
sys.path.append(father_path)
sys.path.append(grader_father)

import json
import re
from urllib.parse import unquote
import hashlib
from lxml.html import etree


def get_md5(url):
    '''
    :param url:
    :return: md5加密
    '''
    issue_md5 = hashlib.md5()
    issue_md5.update(url.encode('utf8'))
    issue_md5 = issue_md5.hexdigest()
    return issue_md5


def parse(self, response):
    # print(response.status)
    if response.status == 403:
        return
    try:
        url = unquote(response.meta['redirect_urls'][0])
    except:
        url = unquote(response.url)
    item = dict()
    item['id'] = get_md5(url)
    try:
        content = json.loads(re.findall('{"is.+}', response.text)[0])['product']
    except:
        item['tag'] = []
        item['online'] = 0
        item['available'] = 1
        item['salable'] = '不在线'
        item['finished'] = True
        yield item
        return
    doc = etree.HTML(response.text)
    try:
        item['review_count'] = content['commentNumber']
    except:
        item['review_count'] = 0
    item['bad_review_count'] = 0
    item['landlord_review_count'] = doc.xpath('//div[contains(text(),"评价")]/../div[2]/text()')[0]
    try:
        item['rating'] = content['starRating'] / 10
    except:
        item['rating'] = 0
    item['third_id'] = content['productId']
    item['online'] = 1
    item['available'] = 1
    item['salable'] = ''
    item['tag'] = [i['tagName'] for i in content['productTagList'] if '分' not in i.get('tagName', '分')]
    item['price'] = doc.xpath('//span[@itemprop="price"]/text()')[0]
    item['finished'] = True
    yield item

if __name__ == '__main__':
    pass